#ifndef INDEXER_H_
#define INDEXER_H_

#include <sstream>
#include <fstream>
#include <stdlib.h>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/timeb.h>

#include "utils.h"
#include "reader.h"
#include "index_compressor.h"
#include "segment.h"
#include "hash_table.h"
#include "conf.h"

const int LBUFFERLEN = 1024;
const int GBUFFERLEN = 4096;
const int MAX_DOC_NUM = 2560000;
const int MAX_WORDSNUM_SIZE = 300000;
//const int PATH_MAX_LEN = 255;
const int PARTITION_SIZE = 256;

struct SBuffer {
  int pos;
  int size;
  char* pbuffer;
};


enum FieldType {
  T_TITLE, T_CONTENT
};

class Indexer {

public:
  Indexer();
  ~Indexer();

  /** index all data.
   *
   *  @param dir_name  the directory of the test data.
   */
  void index_all(const string& dir_name);

  /** index one file
   *
   *  @param test    the content of one file.
   *  @param doc_seq the seq no. of documents.
   *  @param field   to identify which field we are indexing.
   */
  void index_one(const string& text, int& doc_seq, FieldType field);

  /** write buffer to disk
   *
   *  @param index_buffer  the words' buffer
   *  @param id            id of word
   *  @param pstr          pointer to the compressed data
   *  @param len           the length of the compressed data
   *  @param idx_path      the path of temporary inverse index files 
   */
  void write_buffer(SBuffer* index_buffer,
                    int id,
                    const char* pstr,
                    int len,
                    const string& idx_path);

  /** flush all words' buffer to disk
   *
   *  @param index_buffer   the words' buffer
   *  @param idx_path       the path of temporary inverse index files 
   */
  void flush(SBuffer* index_buffer, const string& idx_path);

  /** merge all temporary index to the final inverse index
   *  
   *  @param idx_file     the final inverse index file name
   *  @param idx_path     the path of temporary inverse index files 
   */
  void merge(const string& idx_file, const string& idx_path);

  /** save document information to disk
   *
   *  @param file   the file name to store the information
   */
  void save_doc_info(const string& file) const;


  /** complete the final work
   *  do some saving word.
   */
  void complete() const;

  /** show the statistical information.
   *  segment words number, document number and used time.
   */
  void stat() const;

private:

  int word_seq_;
  int pre_word_seq_;
  int doc_seq_;
  int pre_doc_seq_c_[MAX_WORDSNUM_SIZE];
  int pre_doc_seq_t_[MAX_WORDSNUM_SIZE];
  SBuffer c_index_buffer_[MAX_WORDSNUM_SIZE];
  SBuffer t_index_buffer_[MAX_WORDSNUM_SIZE];
  string cidx_path_;
  string cidx_file_;
  string tidx_path_;
  string tidx_file_;
  int used_time_;
  int total_doc_len_;

  int* index_offset_;
  IndexCompressor* index_compr_;
  HashTable* words_ht_;
  DocInfo* doc_info_;
  Segment* seg_;
  //ofstream fout;
};

#endif
